{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5c149523",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "from tqdm import tqdm\n",
    "from collections import defaultdict\n",
    "import numpy as np\n",
    "from typing import *\n",
    "import time\n",
    "import copy\n",
    "import warnings\n",
    "\n",
    "from data.openai import *\n",
    "from data.generation import *\n",
    "from data.finetune import *\n",
    "from data.inference import *\n",
    "from data.io import *\n",
    "from data.evaluation import *\n",
    "from data.split import *\n",
    "\n",
    "from utils.paths import *\n",
    "from utils.metadata import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a7ad01d7",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import openai\n",
    "openai.api_key = \"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\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5132efaa",
   "metadata": {},
   "outputs": [],
   "source": [
    "BASE_MODELS = [\n",
    "    \"ada\", \"babbage\", \"curie\",\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d6264f98",
   "metadata": {},
   "outputs": [],
   "source": [
    "DATASETS = [\n",
    "    \"single_eq\", \"addsub\", \"multiarith\", \"gsm8k\", \"aqua\", \"svamp\",\n",
    "    \"date_understanding\", \"coin_flip\",\n",
    "    \"tracking_shuffled_objects\", \"last_letter_concatenation\",\n",
    "    \"commonsense_qa\", \"strategy_qa\",\n",
    "]\n",
    "datasets = DATASETS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "51db5a65",
   "metadata": {},
   "outputs": [],
   "source": [
    "DATASET_NAMES = {\n",
    "    \"single_eq\": \"SingleEq\",\n",
    "    \"addsub\": \"AddSub\",\n",
    "    \"multiarith\": \"MultiArith\",\n",
    "    \"gsm8k\": \"GSM8K\",\n",
    "    \"aqua\": \"AQUA\",\n",
    "    \"svamp\": \"SVAMP\",\n",
    "    \"commonsense_qa\": \"Common\",  # SenseQA\n",
    "    \"strategy_qa\": \"Strategy\",  # QA\n",
    "    \"date_understanding\": \"Date\",  # Understanding\n",
    "    \"tracking_shuffled_objects\": \"Shuffled\",  # Objects\n",
    "    \"last_letter_concatenation\": \"Last Letter\",  # (4 words)\n",
    "    \"coin_flip\": \"Coin Flip\",  # (4 times)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "ebdadb37",
   "metadata": {},
   "outputs": [],
   "source": [
    "FULL_DATASET_NAMES = { \"single_eq\": \"SingleEq\", \"addsub\": \"AddSub\", \"multiarith\": \"MultiArith\", \"gsm8k\": \"GSM8K\", \"aqua\": \"AQUA\", \"svamp\": \"SVAMP\", \"commonsense_qa\": \"CommonSenseQA\", \"strategy_qa\": \"StrategyQA\", \"date_understanding\": \"Date Understanding\", \"tracking_shuffled_objects\": \"Shuffled Objects\", \"last_letter_concatenation\": \"Last Letter (4 words)\", \"coin_flip\": \"Coin Flip (4 words)\"}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "83d28f2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "METHOD_NAMES = {\n",
    "    \"zs\": \"Zero-shot\",\n",
    "    \"ft\": \"Fine-tune\",\n",
    "    \"zs_cot\": \"Zero-shot-CoT\",\n",
    "    \"ft_cot\": \"Fine-tune-CoT\",\n",
    "    \"ft_cot_008shot\": \"8-shot Ft-CoT\",\n",
    "    \"ft_cot_032shot\": \"32-shot Ft-CoT\",\n",
    "    \"ft_cot_128shot\": \"128-shot Ft-CoT\",\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0951517f",
   "metadata": {},
   "source": [
    "# Student Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "74137bab",
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_whitespaces(string):\n",
    "    lines = string.split(\"\\n\")\n",
    "    lines = [line.strip() for line in lines]\n",
    "    lines = filter(lambda a:a, lines)\n",
    "    return \" \".join(lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "695da34f",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def write_samples(completion_data, dataset_key, f):\n",
    "    for i, samples in completion_data.items():\n",
    "        if samples:\n",
    "            s = samples[0]\n",
    "            q = s[\"question\"]\n",
    "            a = s[\"answer\"]\n",
    "            c = s[\"completion\"]\n",
    "            if c.find(\"END\") != -1:\n",
    "                c = c[:c.find(\"END\")]\n",
    "\n",
    "            clean_answer = cleanse_answer(a, dataset_key)\n",
    "            clean_prediction, prediction_candidates = cleanse_prediction(\n",
    "                c, dataset_key, answer_prefix=\"-->\", stop_sequence=\"END\")\n",
    "            \n",
    "            correct = (clean_answer == clean_prediction)\n",
    "            correctness = \"CORRECT  \" if correct else \"INCORRECT\"\n",
    "            complete = s[\"finish_reason\"] == \"stop\" or s[\"completion\"] != -1\n",
    "            finishedness = \"COMPLETE  \" if complete else \"INCOMPLETE\"\n",
    "            \n",
    "            f.write(\"### {} #{:d} (`{} - {}`)\\n\".format(dataset_key.upper(), i, finishedness, correctness))\n",
    "            f.write(\"- **Question**: {}\\n\".format(q))\n",
    "            f.write(\"- **Answer**: {} (`{}`)\\n\".format(a, clean_answer))\n",
    "            f.write(\"- **Completion**: {}\\n\".format(remove_whitespaces(c)))\n",
    "            f.write(\"- **Prediction**: `{}`\\n\".format(clean_prediction))\n",
    "            if len(prediction_candidates) >= 2:\n",
    "                f.write(\"- *Candidates: \")\n",
    "                for i, candidate in enumerate(prediction_candidates):\n",
    "                    f.write(\"`{}`, \".format(candidate))\n",
    "                f.write(\"\\n\")\n",
    "        f.write(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "97880f3f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Samples saved:\n",
      "manual_inspection/finetune_cot_ada_single_eq.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_ada_addsub.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_ada_multiarith.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_ada_gsm8k.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_ada_aqua.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_ada_svamp.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_ada_date_understanding.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_ada_coin_flip.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_ada_tracking_shuffled_objects.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_ada_last_letter_concatenation.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_ada_commonsense_qa.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_ada_strategy_qa.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_babbage_single_eq.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_babbage_addsub.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_babbage_multiarith.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_babbage_gsm8k.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_babbage_aqua.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_babbage_svamp.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_babbage_date_understanding.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_babbage_coin_flip.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_babbage_tracking_shuffled_objects.md\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/itsnamgyu/teach-step-by-step/data/evaluation.py:160: RuntimeWarning: invalid value encountered in long_scalars\n",
      "  \"accuracy_without_prefix\": (~contains_prefix & correct).sum() / (~contains_prefix).sum(),\n",
      "/home/itsnamgyu/teach-step-by-step/data/evaluation.py:175: RuntimeWarning: invalid value encountered in long_scalars\n",
      "  \"accuracy_when_incomplete\": (~complete & correct).sum() / (~complete).sum(),\n",
      "/home/itsnamgyu/teach-step-by-step/data/evaluation.py:160: RuntimeWarning: invalid value encountered in long_scalars\n",
      "  \"accuracy_without_prefix\": (~contains_prefix & correct).sum() / (~contains_prefix).sum(),\n",
      "/home/itsnamgyu/teach-step-by-step/data/evaluation.py:175: RuntimeWarning: invalid value encountered in long_scalars\n",
      "  \"accuracy_when_incomplete\": (~complete & correct).sum() / (~complete).sum(),\n",
      "/home/itsnamgyu/teach-step-by-step/data/evaluation.py:160: RuntimeWarning: invalid value encountered in long_scalars\n",
      "  \"accuracy_without_prefix\": (~contains_prefix & correct).sum() / (~contains_prefix).sum(),\n",
      "/home/itsnamgyu/teach-step-by-step/data/evaluation.py:175: RuntimeWarning: invalid value encountered in long_scalars\n",
      "  \"accuracy_when_incomplete\": (~complete & correct).sum() / (~complete).sum(),\n",
      "/home/itsnamgyu/teach-step-by-step/data/evaluation.py:160: RuntimeWarning: invalid value encountered in long_scalars\n",
      "  \"accuracy_without_prefix\": (~contains_prefix & correct).sum() / (~contains_prefix).sum(),\n",
      "/home/itsnamgyu/teach-step-by-step/data/evaluation.py:175: RuntimeWarning: invalid value encountered in long_scalars\n",
      "  \"accuracy_when_incomplete\": (~complete & correct).sum() / (~complete).sum(),\n",
      "/home/itsnamgyu/teach-step-by-step/data/evaluation.py:160: RuntimeWarning: invalid value encountered in long_scalars\n",
      "  \"accuracy_without_prefix\": (~contains_prefix & correct).sum() / (~contains_prefix).sum(),\n",
      "/home/itsnamgyu/teach-step-by-step/data/evaluation.py:175: RuntimeWarning: invalid value encountered in long_scalars\n",
      "  \"accuracy_when_incomplete\": (~complete & correct).sum() / (~complete).sum(),\n",
      "/home/itsnamgyu/teach-step-by-step/data/evaluation.py:160: RuntimeWarning: invalid value encountered in long_scalars\n",
      "  \"accuracy_without_prefix\": (~contains_prefix & correct).sum() / (~contains_prefix).sum(),\n",
      "/home/itsnamgyu/teach-step-by-step/data/evaluation.py:175: RuntimeWarning: invalid value encountered in long_scalars\n",
      "  \"accuracy_when_incomplete\": (~complete & correct).sum() / (~complete).sum(),\n",
      "/home/itsnamgyu/teach-step-by-step/data/evaluation.py:160: RuntimeWarning: invalid value encountered in long_scalars\n",
      "  \"accuracy_without_prefix\": (~contains_prefix & correct).sum() / (~contains_prefix).sum(),\n",
      "/home/itsnamgyu/teach-step-by-step/data/evaluation.py:175: RuntimeWarning: invalid value encountered in long_scalars\n",
      "  \"accuracy_when_incomplete\": (~complete & correct).sum() / (~complete).sum(),\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Samples saved:\n",
      "manual_inspection/finetune_cot_babbage_last_letter_concatenation.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_babbage_commonsense_qa.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_babbage_strategy_qa.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_curie_single_eq.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_curie_addsub.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_curie_multiarith.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_curie_gsm8k.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_curie_aqua.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_curie_svamp.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_curie_date_understanding.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_curie_coin_flip.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_curie_tracking_shuffled_objects.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_curie_last_letter_concatenation.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_curie_commonsense_qa.md\n",
      "Samples saved:\n",
      "manual_inspection/finetune_cot_curie_strategy_qa.md\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/itsnamgyu/teach-step-by-step/data/evaluation.py:160: RuntimeWarning: invalid value encountered in long_scalars\n",
      "  \"accuracy_without_prefix\": (~contains_prefix & correct).sum() / (~contains_prefix).sum(),\n",
      "/home/itsnamgyu/teach-step-by-step/data/evaluation.py:175: RuntimeWarning: invalid value encountered in long_scalars\n",
      "  \"accuracy_when_incomplete\": (~complete & correct).sum() / (~complete).sum(),\n",
      "/home/itsnamgyu/teach-step-by-step/data/evaluation.py:160: RuntimeWarning: invalid value encountered in long_scalars\n",
      "  \"accuracy_without_prefix\": (~contains_prefix & correct).sum() / (~contains_prefix).sum(),\n",
      "/home/itsnamgyu/teach-step-by-step/data/evaluation.py:175: RuntimeWarning: invalid value encountered in long_scalars\n",
      "  \"accuracy_when_incomplete\": (~complete & correct).sum() / (~complete).sum(),\n",
      "/home/itsnamgyu/teach-step-by-step/data/evaluation.py:160: RuntimeWarning: invalid value encountered in long_scalars\n",
      "  \"accuracy_without_prefix\": (~contains_prefix & correct).sum() / (~contains_prefix).sum(),\n",
      "/home/itsnamgyu/teach-step-by-step/data/evaluation.py:175: RuntimeWarning: invalid value encountered in long_scalars\n",
      "  \"accuracy_when_incomplete\": (~complete & correct).sum() / (~complete).sum(),\n"
     ]
    }
   ],
   "source": [
    "for base_model_key in BASE_MODELS:\n",
    "    for dataset_key in DATASETS:\n",
    "        template = \"special\"\n",
    "        completion_key  = \"finetune_cot\"\n",
    "        file_key = \"zs_cot_{}_{}_train\".format(template, dataset_key)\n",
    "        model_key = \"{}_{}\".format(base_model_key, file_key)\n",
    "\n",
    "        completion_data = load_completion_data(completion_key, dataset_key, model_key)\n",
    "        evaluation = evaluate_completions(completion_data, dataset_key)\n",
    "        metrics = get_evaluation_metrics(evaluation)\n",
    "        \n",
    "        basename = \"finetune_cot_{}_{}.md\".format(base_model_key, dataset_key)\n",
    "        path = \"manual_inspection\"\n",
    "        path = os.path.join(path, basename)\n",
    "        os.makedirs(os.path.dirname(path), exist_ok=True)\n",
    "        with open(path, \"w\") as f:\n",
    "            f.write(\"## Metrics\\n\")\n",
    "            f.write(\"```\\n\")\n",
    "            for key, value in metrics.items():\n",
    "                f.write(\"{:40s}: {:6.2f}%\\n\".format(key, value * 100))\n",
    "            f.write(\"```\\n\")\n",
    "            f.write(\"\\n\")\n",
    "            f.write(\"## Samples\\n\")\n",
    "            f.write(\"\\n\")\n",
    "            write_samples(completion_data, dataset_key, f)\n",
    "        print(\"Samples saved:\")\n",
    "        print(path)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "96d0cac4",
   "metadata": {},
   "source": [
    "# Teacher Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "4cb8e1bd",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "single_eq\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/itsnamgyu/teach-step-by-step/data/evaluation.py:159: RuntimeWarning: invalid value encountered in long_scalars\n",
      "  \"accuracy_with_prefix\": (contains_prefix & correct).sum() / contains_prefix.sum(),\n",
      "/home/itsnamgyu/teach-step-by-step/data/evaluation.py:173: RuntimeWarning: invalid value encountered in long_scalars\n",
      "  \"accuracy_when_complete\": (complete & correct).sum() / complete.sum(),\n"
     ]
    }
   ],
   "source": [
    "for dataset_key in DATASETS:\n",
    "    break\n",
    "ho = 1\n",
    " \n",
    "completion_key  = \"zs_cot\"\n",
    "model_key = \"text-davinci-002\"\n",
    "\n",
    "print(dataset_key)\n",
    "\n",
    "completion_data = load_completion_data(completion_key, dataset_key, model_key)\n",
    "evaluation = evaluate_completions(completion_data, dataset_key)\n",
    "metrics = get_evaluation_metrics(evaluation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9c63fa96",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def write_teacher_samples(completion_data, dataset_key, f):\n",
    "    for i, samples in completion_data.items():\n",
    "        if samples:\n",
    "            s = samples[0]\n",
    "            q = s[\"question\"]\n",
    "            reasoning = remove_whitespaces(s[\"reasoning_completion\"])\n",
    "            a = s[\"answer\"]\n",
    "            c = s[\"completion\"]\n",
    "            if c.find(\"END\") != -1:\n",
    "                c = c[:c.find(\"END\")]\n",
    "\n",
    "            clean_answer = cleanse_answer(a, dataset_key)\n",
    "            clean_prediction, prediction_candidates = cleanse_prediction(\n",
    "                c, dataset_key, answer_prefix=\"-->\", stop_sequence=\"END\")\n",
    "\n",
    "            correct = (clean_answer == clean_prediction)\n",
    "            correctness = \"CORRECT  \" if correct else \"INCORRECT\"\n",
    "            complete = s[\"reasoning_finish_reason\"] == \"stop\" or c.find(\"END\") != -1\n",
    "            finishedness = \"COMPLETE  \" if complete else \"INCOMPLETE\"\n",
    "\n",
    "\n",
    "            f.write(\"### {} #{:d} (`{} - {}`)\\n\".format(dataset_key.upper(), i, finishedness, correctness))\n",
    "            f.write(\"- **Question**: {} *Let's think step by step*\\n\".format(q))\n",
    "            f.write(\"- **Reasoning**: {} *Therefore, the answer is*\\n\".format(reasoning))\n",
    "            f.write(\"- **Answer**: {} (`{}`)\\n\".format(a, clean_answer))\n",
    "            f.write(\"- **Completion**: {}\\n\".format(remove_whitespaces(c)))\n",
    "            f.write(\"- **Prediction**: `{}`\\n\".format(clean_prediction))\n",
    "            if len(prediction_candidates) >= 2:\n",
    "                f.write(\"- *Candidates: \")\n",
    "                for i, candidate in enumerate(prediction_candidates):\n",
    "                    f.write(\"`{}`, \".format(candidate))\n",
    "                f.write(\"\\n\")\n",
    "        f.write(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "3f9df722",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Samples saved:\n",
      "manual_inspection/zs_cot_teacher_single_eq_train.md\n",
      "Samples saved:\n",
      "manual_inspection/zs_cot_teacher_addsub_train.md\n",
      "Samples saved:\n",
      "manual_inspection/zs_cot_teacher_multiarith_train.md\n",
      "Samples saved:\n",
      "manual_inspection/zs_cot_teacher_gsm8k_train.md\n",
      "Samples saved:\n",
      "manual_inspection/zs_cot_teacher_aqua_train.md\n",
      "Samples saved:\n",
      "manual_inspection/zs_cot_teacher_svamp_train.md\n",
      "Samples saved:\n",
      "manual_inspection/zs_cot_teacher_date_understanding_train.md\n",
      "Samples saved:\n",
      "manual_inspection/zs_cot_teacher_coin_flip_train.md\n",
      "Samples saved:\n",
      "manual_inspection/zs_cot_teacher_tracking_shuffled_objects_train.md\n",
      "Samples saved:\n",
      "manual_inspection/zs_cot_teacher_last_letter_concatenation_train.md\n",
      "Samples saved:\n",
      "manual_inspection/zs_cot_teacher_commonsense_qa_train.md\n",
      "Samples saved:\n",
      "manual_inspection/zs_cot_teacher_strategy_qa_train.md\n"
     ]
    }
   ],
   "source": [
    "for dataset_key in DATASETS:\n",
    "    completion_key  = \"zs_cot\"\n",
    "    model_key = \"text-davinci-002\"\n",
    "    train_indices, test_indices = get_train_test_indices(dataset_key)\n",
    "\n",
    "    completion_data = load_completion_data(completion_key, dataset_key, model_key)\n",
    "    evaluation = evaluate_completions(completion_data, dataset_key, template=None, indices=train_indices)\n",
    "    metrics = get_evaluation_metrics(evaluation)\n",
    "\n",
    "    basename = \"zs_cot_teacher_{}_train.md\".format(dataset_key)\n",
    "    path = \"manual_inspection\"\n",
    "    path = os.path.join(path, basename)\n",
    "    os.makedirs(os.path.dirname(path), exist_ok=True)\n",
    "    with open(path, \"w\") as f:\n",
    "        f.write(\"## Metrics\\n\")\n",
    "        f.write(\"```\\n\")\n",
    "        for key, value in metrics.items():\n",
    "            f.write(\"{:40s}: {:6.2f}%\\n\".format(key, value * 100))\n",
    "        f.write(\"```\\n\")\n",
    "        f.write(\"\\n\")\n",
    "        f.write(\"## Samples\\n\")\n",
    "        f.write(\"\\n\")\n",
    "        write_teacher_samples(completion_data, dataset_key, f)\n",
    "    print(\"Samples saved:\")\n",
    "    print(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "114da143",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b52c4348",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "822cddda",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a43e6c2",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
